{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'1.10.0'"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.examples.tutorials.mnist import input_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-4-f3e05b6b63fd>:1: read_data_sets (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n",
      "WARNING:tensorflow:From /home/david/.local/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:260: maybe_download (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please write your own downloading logic.\n",
      "WARNING:tensorflow:From /home/david/.local/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:262: extract_images (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting ./train-images-idx3-ubyte.gz\n",
      "WARNING:tensorflow:From /home/david/.local/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:267: extract_labels (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting ./train-labels-idx1-ubyte.gz\n",
      "WARNING:tensorflow:From /home/david/.local/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:110: dense_to_one_hot (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.one_hot on tensors.\n",
      "Extracting ./t10k-images-idx3-ubyte.gz\n",
      "Extracting ./t10k-labels-idx1-ubyte.gz\n",
      "WARNING:tensorflow:From /home/david/.local/lib/python3.6/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:290: DataSet.__init__ (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n",
      "Extracting ./train-images-idx3-ubyte.gz\n",
      "Extracting ./train-labels-idx1-ubyte.gz\n",
      "Extracting ./t10k-images-idx3-ubyte.gz\n",
      "Extracting ./t10k-labels-idx1-ubyte.gz\n"
     ]
    }
   ],
   "source": [
    "mnist = input_data.read_data_sets('.', one_hot=True)\n",
    "mnist2 = input_data.read_data_sets('.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0. 0. 0. 0. 0. 0. 0. 1. 0. 0.]\n",
      " [0. 0. 0. 1. 0. 0. 0. 0. 0. 0.]]\n",
      "[7 3]\n"
     ]
    }
   ],
   "source": [
    "print(mnist.train.labels[:2])\n",
    "print(mnist2.train.labels[:2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-6-bcade7216fdd>:12: softmax_cross_entropy_with_logits (from tensorflow.python.ops.nn_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "\n",
      "Future major versions of TensorFlow will allow gradients to flow\n",
      "into the labels input on backprop by default.\n",
      "\n",
      "See @{tf.nn.softmax_cross_entropy_with_logits_v2}.\n",
      "\n",
      "WARNING:tensorflow:From <ipython-input-6-bcade7216fdd>:17: arg_max (from tensorflow.python.ops.gen_math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `argmax` instead\n"
     ]
    }
   ],
   "source": [
    "learning_rate = tf.placeholder(tf.float32)\n",
    "\n",
    "x = tf.placeholder(tf.float32, [None, 784], name='x')\n",
    "W = tf.Variable(tf.truncated_normal([784, 10]), name='weight')\n",
    "b = tf.Variable(tf.zeros([10]), name='bias')\n",
    "\n",
    "logits = tf.matmul(x, W) + b\n",
    "\n",
    "y = tf.placeholder(tf.float32, [None, 10], name='y')\n",
    "\n",
    "cross_entropy = tf.reduce_mean(\n",
    "    tf.nn.softmax_cross_entropy_with_logits(labels=y, logits=logits))\n",
    "\n",
    "train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(cross_entropy)\n",
    "\n",
    "correct_prediction = tf.equal(tf.arg_max(y,1), tf.arg_max(logits, 1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "grap = tf.get_default_graph()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.framework.ops.Graph at 0x7f3a304ffa20>"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grap"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "node {\n",
       "  name: \"Placeholder\"\n",
       "  op: \"Placeholder\"\n",
       "  attr {\n",
       "    key: \"dtype\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"shape\"\n",
       "    value {\n",
       "      shape {\n",
       "        unknown_rank: true\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"x\"\n",
       "  op: \"Placeholder\"\n",
       "  attr {\n",
       "    key: \"dtype\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"shape\"\n",
       "    value {\n",
       "      shape {\n",
       "        dim {\n",
       "          size: -1\n",
       "        }\n",
       "        dim {\n",
       "          size: 784\n",
       "        }\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"truncated_normal/shape\"\n",
       "  op: \"Const\"\n",
       "  attr {\n",
       "    key: \"dtype\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"value\"\n",
       "    value {\n",
       "      tensor {\n",
       "        dtype: DT_INT32\n",
       "        tensor_shape {\n",
       "          dim {\n",
       "            size: 2\n",
       "          }\n",
       "        }\n",
       "        tensor_content: \"\\020\\003\\000\\000\\n\\000\\000\\000\"\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"truncated_normal/mean\"\n",
       "  op: \"Const\"\n",
       "  attr {\n",
       "    key: \"dtype\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"value\"\n",
       "    value {\n",
       "      tensor {\n",
       "        dtype: DT_FLOAT\n",
       "        tensor_shape {\n",
       "        }\n",
       "        float_val: 0.0\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"truncated_normal/stddev\"\n",
       "  op: \"Const\"\n",
       "  attr {\n",
       "    key: \"dtype\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"value\"\n",
       "    value {\n",
       "      tensor {\n",
       "        dtype: DT_FLOAT\n",
       "        tensor_shape {\n",
       "        }\n",
       "        float_val: 1.0\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"truncated_normal/TruncatedNormal\"\n",
       "  op: \"TruncatedNormal\"\n",
       "  input: \"truncated_normal/shape\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"dtype\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"seed\"\n",
       "    value {\n",
       "      i: 0\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"seed2\"\n",
       "    value {\n",
       "      i: 0\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"truncated_normal/mul\"\n",
       "  op: \"Mul\"\n",
       "  input: \"truncated_normal/TruncatedNormal\"\n",
       "  input: \"truncated_normal/stddev\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"truncated_normal\"\n",
       "  op: \"Add\"\n",
       "  input: \"truncated_normal/mul\"\n",
       "  input: \"truncated_normal/mean\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"weight\"\n",
       "  op: \"VariableV2\"\n",
       "  attr {\n",
       "    key: \"container\"\n",
       "    value {\n",
       "      s: \"\"\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"dtype\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"shape\"\n",
       "    value {\n",
       "      shape {\n",
       "        dim {\n",
       "          size: 784\n",
       "        }\n",
       "        dim {\n",
       "          size: 10\n",
       "        }\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"shared_name\"\n",
       "    value {\n",
       "      s: \"\"\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"weight/Assign\"\n",
       "  op: \"Assign\"\n",
       "  input: \"weight\"\n",
       "  input: \"truncated_normal\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"_class\"\n",
       "    value {\n",
       "      list {\n",
       "        s: \"loc:@weight\"\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"use_locking\"\n",
       "    value {\n",
       "      b: true\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"validate_shape\"\n",
       "    value {\n",
       "      b: true\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"weight/read\"\n",
       "  op: \"Identity\"\n",
       "  input: \"weight\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"_class\"\n",
       "    value {\n",
       "      list {\n",
       "        s: \"loc:@weight\"\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"zeros\"\n",
       "  op: \"Const\"\n",
       "  attr {\n",
       "    key: \"dtype\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"value\"\n",
       "    value {\n",
       "      tensor {\n",
       "        dtype: DT_FLOAT\n",
       "        tensor_shape {\n",
       "          dim {\n",
       "            size: 10\n",
       "          }\n",
       "        }\n",
       "        float_val: 0.0\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"bias\"\n",
       "  op: \"VariableV2\"\n",
       "  attr {\n",
       "    key: \"container\"\n",
       "    value {\n",
       "      s: \"\"\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"dtype\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"shape\"\n",
       "    value {\n",
       "      shape {\n",
       "        dim {\n",
       "          size: 10\n",
       "        }\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"shared_name\"\n",
       "    value {\n",
       "      s: \"\"\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"bias/Assign\"\n",
       "  op: \"Assign\"\n",
       "  input: \"bias\"\n",
       "  input: \"zeros\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"_class\"\n",
       "    value {\n",
       "      list {\n",
       "        s: \"loc:@bias\"\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"use_locking\"\n",
       "    value {\n",
       "      b: true\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"validate_shape\"\n",
       "    value {\n",
       "      b: true\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"bias/read\"\n",
       "  op: \"Identity\"\n",
       "  input: \"bias\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"_class\"\n",
       "    value {\n",
       "      list {\n",
       "        s: \"loc:@bias\"\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"MatMul\"\n",
       "  op: \"MatMul\"\n",
       "  input: \"x\"\n",
       "  input: \"weight/read\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"transpose_a\"\n",
       "    value {\n",
       "      b: false\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"transpose_b\"\n",
       "    value {\n",
       "      b: false\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"add\"\n",
       "  op: \"Add\"\n",
       "  input: \"MatMul\"\n",
       "  input: \"bias/read\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"y\"\n",
       "  op: \"Placeholder\"\n",
       "  attr {\n",
       "    key: \"dtype\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"shape\"\n",
       "    value {\n",
       "      shape {\n",
       "        dim {\n",
       "          size: -1\n",
       "        }\n",
       "        dim {\n",
       "          size: 10\n",
       "        }\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"softmax_cross_entropy_with_logits_sg/labels_stop_gradient\"\n",
       "  op: \"StopGradient\"\n",
       "  input: \"y\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"softmax_cross_entropy_with_logits_sg/Rank\"\n",
       "  op: \"Const\"\n",
       "  attr {\n",
       "    key: \"dtype\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"value\"\n",
       "    value {\n",
       "      tensor {\n",
       "        dtype: DT_INT32\n",
       "        tensor_shape {\n",
       "        }\n",
       "        int_val: 2\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"softmax_cross_entropy_with_logits_sg/Shape\"\n",
       "  op: \"Shape\"\n",
       "  input: \"add\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"out_type\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"softmax_cross_entropy_with_logits_sg/Rank_1\"\n",
       "  op: \"Const\"\n",
       "  attr {\n",
       "    key: \"dtype\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"value\"\n",
       "    value {\n",
       "      tensor {\n",
       "        dtype: DT_INT32\n",
       "        tensor_shape {\n",
       "        }\n",
       "        int_val: 2\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"softmax_cross_entropy_with_logits_sg/Shape_1\"\n",
       "  op: \"Shape\"\n",
       "  input: \"add\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"out_type\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"softmax_cross_entropy_with_logits_sg/Sub/y\"\n",
       "  op: \"Const\"\n",
       "  attr {\n",
       "    key: \"dtype\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"value\"\n",
       "    value {\n",
       "      tensor {\n",
       "        dtype: DT_INT32\n",
       "        tensor_shape {\n",
       "        }\n",
       "        int_val: 1\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"softmax_cross_entropy_with_logits_sg/Sub\"\n",
       "  op: \"Sub\"\n",
       "  input: \"softmax_cross_entropy_with_logits_sg/Rank_1\"\n",
       "  input: \"softmax_cross_entropy_with_logits_sg/Sub/y\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"softmax_cross_entropy_with_logits_sg/Slice/begin\"\n",
       "  op: \"Pack\"\n",
       "  input: \"softmax_cross_entropy_with_logits_sg/Sub\"\n",
       "  attr {\n",
       "    key: \"N\"\n",
       "    value {\n",
       "      i: 1\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"axis\"\n",
       "    value {\n",
       "      i: 0\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"softmax_cross_entropy_with_logits_sg/Slice/size\"\n",
       "  op: \"Const\"\n",
       "  attr {\n",
       "    key: \"dtype\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"value\"\n",
       "    value {\n",
       "      tensor {\n",
       "        dtype: DT_INT32\n",
       "        tensor_shape {\n",
       "          dim {\n",
       "            size: 1\n",
       "          }\n",
       "        }\n",
       "        int_val: 1\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"softmax_cross_entropy_with_logits_sg/Slice\"\n",
       "  op: \"Slice\"\n",
       "  input: \"softmax_cross_entropy_with_logits_sg/Shape_1\"\n",
       "  input: \"softmax_cross_entropy_with_logits_sg/Slice/begin\"\n",
       "  input: \"softmax_cross_entropy_with_logits_sg/Slice/size\"\n",
       "  attr {\n",
       "    key: \"Index\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"softmax_cross_entropy_with_logits_sg/concat/values_0\"\n",
       "  op: \"Const\"\n",
       "  attr {\n",
       "    key: \"dtype\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"value\"\n",
       "    value {\n",
       "      tensor {\n",
       "        dtype: DT_INT32\n",
       "        tensor_shape {\n",
       "          dim {\n",
       "            size: 1\n",
       "          }\n",
       "        }\n",
       "        int_val: -1\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"softmax_cross_entropy_with_logits_sg/concat/axis\"\n",
       "  op: \"Const\"\n",
       "  attr {\n",
       "    key: \"dtype\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"value\"\n",
       "    value {\n",
       "      tensor {\n",
       "        dtype: DT_INT32\n",
       "        tensor_shape {\n",
       "        }\n",
       "        int_val: 0\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"softmax_cross_entropy_with_logits_sg/concat\"\n",
       "  op: \"ConcatV2\"\n",
       "  input: \"softmax_cross_entropy_with_logits_sg/concat/values_0\"\n",
       "  input: \"softmax_cross_entropy_with_logits_sg/Slice\"\n",
       "  input: \"softmax_cross_entropy_with_logits_sg/concat/axis\"\n",
       "  attr {\n",
       "    key: \"N\"\n",
       "    value {\n",
       "      i: 2\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"Tidx\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"softmax_cross_entropy_with_logits_sg/Reshape\"\n",
       "  op: \"Reshape\"\n",
       "  input: \"add\"\n",
       "  input: \"softmax_cross_entropy_with_logits_sg/concat\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"Tshape\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"softmax_cross_entropy_with_logits_sg/Rank_2\"\n",
       "  op: \"Const\"\n",
       "  attr {\n",
       "    key: \"dtype\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"value\"\n",
       "    value {\n",
       "      tensor {\n",
       "        dtype: DT_INT32\n",
       "        tensor_shape {\n",
       "        }\n",
       "        int_val: 2\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"softmax_cross_entropy_with_logits_sg/Shape_2\"\n",
       "  op: \"Shape\"\n",
       "  input: \"softmax_cross_entropy_with_logits_sg/labels_stop_gradient\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"out_type\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"softmax_cross_entropy_with_logits_sg/Sub_1/y\"\n",
       "  op: \"Const\"\n",
       "  attr {\n",
       "    key: \"dtype\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"value\"\n",
       "    value {\n",
       "      tensor {\n",
       "        dtype: DT_INT32\n",
       "        tensor_shape {\n",
       "        }\n",
       "        int_val: 1\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"softmax_cross_entropy_with_logits_sg/Sub_1\"\n",
       "  op: \"Sub\"\n",
       "  input: \"softmax_cross_entropy_with_logits_sg/Rank_2\"\n",
       "  input: \"softmax_cross_entropy_with_logits_sg/Sub_1/y\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"softmax_cross_entropy_with_logits_sg/Slice_1/begin\"\n",
       "  op: \"Pack\"\n",
       "  input: \"softmax_cross_entropy_with_logits_sg/Sub_1\"\n",
       "  attr {\n",
       "    key: \"N\"\n",
       "    value {\n",
       "      i: 1\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"axis\"\n",
       "    value {\n",
       "      i: 0\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"softmax_cross_entropy_with_logits_sg/Slice_1/size\"\n",
       "  op: \"Const\"\n",
       "  attr {\n",
       "    key: \"dtype\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"value\"\n",
       "    value {\n",
       "      tensor {\n",
       "        dtype: DT_INT32\n",
       "        tensor_shape {\n",
       "          dim {\n",
       "            size: 1\n",
       "          }\n",
       "        }\n",
       "        int_val: 1\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"softmax_cross_entropy_with_logits_sg/Slice_1\"\n",
       "  op: \"Slice\"\n",
       "  input: \"softmax_cross_entropy_with_logits_sg/Shape_2\"\n",
       "  input: \"softmax_cross_entropy_with_logits_sg/Slice_1/begin\"\n",
       "  input: \"softmax_cross_entropy_with_logits_sg/Slice_1/size\"\n",
       "  attr {\n",
       "    key: \"Index\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"softmax_cross_entropy_with_logits_sg/concat_1/values_0\"\n",
       "  op: \"Const\"\n",
       "  attr {\n",
       "    key: \"dtype\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"value\"\n",
       "    value {\n",
       "      tensor {\n",
       "        dtype: DT_INT32\n",
       "        tensor_shape {\n",
       "          dim {\n",
       "            size: 1\n",
       "          }\n",
       "        }\n",
       "        int_val: -1\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"softmax_cross_entropy_with_logits_sg/concat_1/axis\"\n",
       "  op: \"Const\"\n",
       "  attr {\n",
       "    key: \"dtype\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"value\"\n",
       "    value {\n",
       "      tensor {\n",
       "        dtype: DT_INT32\n",
       "        tensor_shape {\n",
       "        }\n",
       "        int_val: 0\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"softmax_cross_entropy_with_logits_sg/concat_1\"\n",
       "  op: \"ConcatV2\"\n",
       "  input: \"softmax_cross_entropy_with_logits_sg/concat_1/values_0\"\n",
       "  input: \"softmax_cross_entropy_with_logits_sg/Slice_1\"\n",
       "  input: \"softmax_cross_entropy_with_logits_sg/concat_1/axis\"\n",
       "  attr {\n",
       "    key: \"N\"\n",
       "    value {\n",
       "      i: 2\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"Tidx\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"softmax_cross_entropy_with_logits_sg/Reshape_1\"\n",
       "  op: \"Reshape\"\n",
       "  input: \"softmax_cross_entropy_with_logits_sg/labels_stop_gradient\"\n",
       "  input: \"softmax_cross_entropy_with_logits_sg/concat_1\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"Tshape\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"softmax_cross_entropy_with_logits_sg\"\n",
       "  op: \"SoftmaxCrossEntropyWithLogits\"\n",
       "  input: \"softmax_cross_entropy_with_logits_sg/Reshape\"\n",
       "  input: \"softmax_cross_entropy_with_logits_sg/Reshape_1\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"softmax_cross_entropy_with_logits_sg/Sub_2/y\"\n",
       "  op: \"Const\"\n",
       "  attr {\n",
       "    key: \"dtype\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"value\"\n",
       "    value {\n",
       "      tensor {\n",
       "        dtype: DT_INT32\n",
       "        tensor_shape {\n",
       "        }\n",
       "        int_val: 1\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"softmax_cross_entropy_with_logits_sg/Sub_2\"\n",
       "  op: \"Sub\"\n",
       "  input: \"softmax_cross_entropy_with_logits_sg/Rank\"\n",
       "  input: \"softmax_cross_entropy_with_logits_sg/Sub_2/y\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"softmax_cross_entropy_with_logits_sg/Slice_2/begin\"\n",
       "  op: \"Const\"\n",
       "  attr {\n",
       "    key: \"dtype\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"value\"\n",
       "    value {\n",
       "      tensor {\n",
       "        dtype: DT_INT32\n",
       "        tensor_shape {\n",
       "          dim {\n",
       "            size: 1\n",
       "          }\n",
       "        }\n",
       "        int_val: 0\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"softmax_cross_entropy_with_logits_sg/Slice_2/size\"\n",
       "  op: \"Pack\"\n",
       "  input: \"softmax_cross_entropy_with_logits_sg/Sub_2\"\n",
       "  attr {\n",
       "    key: \"N\"\n",
       "    value {\n",
       "      i: 1\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"axis\"\n",
       "    value {\n",
       "      i: 0\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"softmax_cross_entropy_with_logits_sg/Slice_2\"\n",
       "  op: \"Slice\"\n",
       "  input: \"softmax_cross_entropy_with_logits_sg/Shape\"\n",
       "  input: \"softmax_cross_entropy_with_logits_sg/Slice_2/begin\"\n",
       "  input: \"softmax_cross_entropy_with_logits_sg/Slice_2/size\"\n",
       "  attr {\n",
       "    key: \"Index\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"softmax_cross_entropy_with_logits_sg/Reshape_2\"\n",
       "  op: \"Reshape\"\n",
       "  input: \"softmax_cross_entropy_with_logits_sg\"\n",
       "  input: \"softmax_cross_entropy_with_logits_sg/Slice_2\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"Tshape\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"Const\"\n",
       "  op: \"Const\"\n",
       "  attr {\n",
       "    key: \"dtype\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"value\"\n",
       "    value {\n",
       "      tensor {\n",
       "        dtype: DT_INT32\n",
       "        tensor_shape {\n",
       "          dim {\n",
       "            size: 1\n",
       "          }\n",
       "        }\n",
       "        int_val: 0\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"Mean\"\n",
       "  op: \"Mean\"\n",
       "  input: \"softmax_cross_entropy_with_logits_sg/Reshape_2\"\n",
       "  input: \"Const\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"Tidx\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"keep_dims\"\n",
       "    value {\n",
       "      b: false\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/Shape\"\n",
       "  op: \"Const\"\n",
       "  attr {\n",
       "    key: \"dtype\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"value\"\n",
       "    value {\n",
       "      tensor {\n",
       "        dtype: DT_INT32\n",
       "        tensor_shape {\n",
       "          dim {\n",
       "          }\n",
       "        }\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/grad_ys_0\"\n",
       "  op: \"Const\"\n",
       "  attr {\n",
       "    key: \"dtype\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"value\"\n",
       "    value {\n",
       "      tensor {\n",
       "        dtype: DT_FLOAT\n",
       "        tensor_shape {\n",
       "        }\n",
       "        float_val: 1.0\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/Fill\"\n",
       "  op: \"Fill\"\n",
       "  input: \"gradients/Shape\"\n",
       "  input: \"gradients/grad_ys_0\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"index_type\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/Mean_grad/Reshape/shape\"\n",
       "  op: \"Const\"\n",
       "  attr {\n",
       "    key: \"dtype\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"value\"\n",
       "    value {\n",
       "      tensor {\n",
       "        dtype: DT_INT32\n",
       "        tensor_shape {\n",
       "          dim {\n",
       "            size: 1\n",
       "          }\n",
       "        }\n",
       "        int_val: 1\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/Mean_grad/Reshape\"\n",
       "  op: \"Reshape\"\n",
       "  input: \"gradients/Fill\"\n",
       "  input: \"gradients/Mean_grad/Reshape/shape\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"Tshape\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/Mean_grad/Shape\"\n",
       "  op: \"Shape\"\n",
       "  input: \"softmax_cross_entropy_with_logits_sg/Reshape_2\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"out_type\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/Mean_grad/Tile\"\n",
       "  op: \"Tile\"\n",
       "  input: \"gradients/Mean_grad/Reshape\"\n",
       "  input: \"gradients/Mean_grad/Shape\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"Tmultiples\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/Mean_grad/Shape_1\"\n",
       "  op: \"Shape\"\n",
       "  input: \"softmax_cross_entropy_with_logits_sg/Reshape_2\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"out_type\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/Mean_grad/Shape_2\"\n",
       "  op: \"Const\"\n",
       "  attr {\n",
       "    key: \"dtype\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"value\"\n",
       "    value {\n",
       "      tensor {\n",
       "        dtype: DT_INT32\n",
       "        tensor_shape {\n",
       "          dim {\n",
       "          }\n",
       "        }\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/Mean_grad/Const\"\n",
       "  op: \"Const\"\n",
       "  attr {\n",
       "    key: \"dtype\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"value\"\n",
       "    value {\n",
       "      tensor {\n",
       "        dtype: DT_INT32\n",
       "        tensor_shape {\n",
       "          dim {\n",
       "            size: 1\n",
       "          }\n",
       "        }\n",
       "        int_val: 0\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/Mean_grad/Prod\"\n",
       "  op: \"Prod\"\n",
       "  input: \"gradients/Mean_grad/Shape_1\"\n",
       "  input: \"gradients/Mean_grad/Const\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"Tidx\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"keep_dims\"\n",
       "    value {\n",
       "      b: false\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/Mean_grad/Const_1\"\n",
       "  op: \"Const\"\n",
       "  attr {\n",
       "    key: \"dtype\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"value\"\n",
       "    value {\n",
       "      tensor {\n",
       "        dtype: DT_INT32\n",
       "        tensor_shape {\n",
       "          dim {\n",
       "            size: 1\n",
       "          }\n",
       "        }\n",
       "        int_val: 0\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/Mean_grad/Prod_1\"\n",
       "  op: \"Prod\"\n",
       "  input: \"gradients/Mean_grad/Shape_2\"\n",
       "  input: \"gradients/Mean_grad/Const_1\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"Tidx\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"keep_dims\"\n",
       "    value {\n",
       "      b: false\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/Mean_grad/Maximum/y\"\n",
       "  op: \"Const\"\n",
       "  attr {\n",
       "    key: \"dtype\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"value\"\n",
       "    value {\n",
       "      tensor {\n",
       "        dtype: DT_INT32\n",
       "        tensor_shape {\n",
       "        }\n",
       "        int_val: 1\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/Mean_grad/Maximum\"\n",
       "  op: \"Maximum\"\n",
       "  input: \"gradients/Mean_grad/Prod_1\"\n",
       "  input: \"gradients/Mean_grad/Maximum/y\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/Mean_grad/floordiv\"\n",
       "  op: \"FloorDiv\"\n",
       "  input: \"gradients/Mean_grad/Prod\"\n",
       "  input: \"gradients/Mean_grad/Maximum\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/Mean_grad/Cast\"\n",
       "  op: \"Cast\"\n",
       "  input: \"gradients/Mean_grad/floordiv\"\n",
       "  attr {\n",
       "    key: \"DstT\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"SrcT\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/Mean_grad/truediv\"\n",
       "  op: \"RealDiv\"\n",
       "  input: \"gradients/Mean_grad/Tile\"\n",
       "  input: \"gradients/Mean_grad/Cast\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/softmax_cross_entropy_with_logits_sg/Reshape_2_grad/Shape\"\n",
       "  op: \"Shape\"\n",
       "  input: \"softmax_cross_entropy_with_logits_sg\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"out_type\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/softmax_cross_entropy_with_logits_sg/Reshape_2_grad/Reshape\"\n",
       "  op: \"Reshape\"\n",
       "  input: \"gradients/Mean_grad/truediv\"\n",
       "  input: \"gradients/softmax_cross_entropy_with_logits_sg/Reshape_2_grad/Shape\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"Tshape\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/zeros_like\"\n",
       "  op: \"ZerosLike\"\n",
       "  input: \"softmax_cross_entropy_with_logits_sg:1\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/softmax_cross_entropy_with_logits_sg_grad/ExpandDims/dim\"\n",
       "  op: \"Const\"\n",
       "  attr {\n",
       "    key: \"dtype\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"value\"\n",
       "    value {\n",
       "      tensor {\n",
       "        dtype: DT_INT32\n",
       "        tensor_shape {\n",
       "        }\n",
       "        int_val: -1\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/softmax_cross_entropy_with_logits_sg_grad/ExpandDims\"\n",
       "  op: \"ExpandDims\"\n",
       "  input: \"gradients/softmax_cross_entropy_with_logits_sg/Reshape_2_grad/Reshape\"\n",
       "  input: \"gradients/softmax_cross_entropy_with_logits_sg_grad/ExpandDims/dim\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"Tdim\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/softmax_cross_entropy_with_logits_sg_grad/mul\"\n",
       "  op: \"Mul\"\n",
       "  input: \"gradients/softmax_cross_entropy_with_logits_sg_grad/ExpandDims\"\n",
       "  input: \"softmax_cross_entropy_with_logits_sg:1\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/softmax_cross_entropy_with_logits_sg_grad/LogSoftmax\"\n",
       "  op: \"LogSoftmax\"\n",
       "  input: \"softmax_cross_entropy_with_logits_sg/Reshape\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/softmax_cross_entropy_with_logits_sg_grad/Neg\"\n",
       "  op: \"Neg\"\n",
       "  input: \"gradients/softmax_cross_entropy_with_logits_sg_grad/LogSoftmax\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/softmax_cross_entropy_with_logits_sg_grad/ExpandDims_1/dim\"\n",
       "  op: \"Const\"\n",
       "  attr {\n",
       "    key: \"dtype\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"value\"\n",
       "    value {\n",
       "      tensor {\n",
       "        dtype: DT_INT32\n",
       "        tensor_shape {\n",
       "        }\n",
       "        int_val: -1\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/softmax_cross_entropy_with_logits_sg_grad/ExpandDims_1\"\n",
       "  op: \"ExpandDims\"\n",
       "  input: \"gradients/softmax_cross_entropy_with_logits_sg/Reshape_2_grad/Reshape\"\n",
       "  input: \"gradients/softmax_cross_entropy_with_logits_sg_grad/ExpandDims_1/dim\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"Tdim\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/softmax_cross_entropy_with_logits_sg_grad/mul_1\"\n",
       "  op: \"Mul\"\n",
       "  input: \"gradients/softmax_cross_entropy_with_logits_sg_grad/ExpandDims_1\"\n",
       "  input: \"gradients/softmax_cross_entropy_with_logits_sg_grad/Neg\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/softmax_cross_entropy_with_logits_sg_grad/tuple/group_deps\"\n",
       "  op: \"NoOp\"\n",
       "  input: \"^gradients/softmax_cross_entropy_with_logits_sg_grad/mul\"\n",
       "  input: \"^gradients/softmax_cross_entropy_with_logits_sg_grad/mul_1\"\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/softmax_cross_entropy_with_logits_sg_grad/tuple/control_dependency\"\n",
       "  op: \"Identity\"\n",
       "  input: \"gradients/softmax_cross_entropy_with_logits_sg_grad/mul\"\n",
       "  input: \"^gradients/softmax_cross_entropy_with_logits_sg_grad/tuple/group_deps\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"_class\"\n",
       "    value {\n",
       "      list {\n",
       "        s: \"loc:@gradients/softmax_cross_entropy_with_logits_sg_grad/mul\"\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/softmax_cross_entropy_with_logits_sg_grad/tuple/control_dependency_1\"\n",
       "  op: \"Identity\"\n",
       "  input: \"gradients/softmax_cross_entropy_with_logits_sg_grad/mul_1\"\n",
       "  input: \"^gradients/softmax_cross_entropy_with_logits_sg_grad/tuple/group_deps\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"_class\"\n",
       "    value {\n",
       "      list {\n",
       "        s: \"loc:@gradients/softmax_cross_entropy_with_logits_sg_grad/mul_1\"\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/softmax_cross_entropy_with_logits_sg/Reshape_grad/Shape\"\n",
       "  op: \"Shape\"\n",
       "  input: \"add\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"out_type\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/softmax_cross_entropy_with_logits_sg/Reshape_grad/Reshape\"\n",
       "  op: \"Reshape\"\n",
       "  input: \"gradients/softmax_cross_entropy_with_logits_sg_grad/tuple/control_dependency\"\n",
       "  input: \"gradients/softmax_cross_entropy_with_logits_sg/Reshape_grad/Shape\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"Tshape\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/add_grad/Shape\"\n",
       "  op: \"Shape\"\n",
       "  input: \"MatMul\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"out_type\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/add_grad/Shape_1\"\n",
       "  op: \"Const\"\n",
       "  attr {\n",
       "    key: \"dtype\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"value\"\n",
       "    value {\n",
       "      tensor {\n",
       "        dtype: DT_INT32\n",
       "        tensor_shape {\n",
       "          dim {\n",
       "            size: 1\n",
       "          }\n",
       "        }\n",
       "        int_val: 10\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/add_grad/BroadcastGradientArgs\"\n",
       "  op: \"BroadcastGradientArgs\"\n",
       "  input: \"gradients/add_grad/Shape\"\n",
       "  input: \"gradients/add_grad/Shape_1\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/add_grad/Sum\"\n",
       "  op: \"Sum\"\n",
       "  input: \"gradients/softmax_cross_entropy_with_logits_sg/Reshape_grad/Reshape\"\n",
       "  input: \"gradients/add_grad/BroadcastGradientArgs\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"Tidx\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"keep_dims\"\n",
       "    value {\n",
       "      b: false\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/add_grad/Reshape\"\n",
       "  op: \"Reshape\"\n",
       "  input: \"gradients/add_grad/Sum\"\n",
       "  input: \"gradients/add_grad/Shape\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"Tshape\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/add_grad/Sum_1\"\n",
       "  op: \"Sum\"\n",
       "  input: \"gradients/softmax_cross_entropy_with_logits_sg/Reshape_grad/Reshape\"\n",
       "  input: \"gradients/add_grad/BroadcastGradientArgs:1\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"Tidx\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"keep_dims\"\n",
       "    value {\n",
       "      b: false\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/add_grad/Reshape_1\"\n",
       "  op: \"Reshape\"\n",
       "  input: \"gradients/add_grad/Sum_1\"\n",
       "  input: \"gradients/add_grad/Shape_1\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"Tshape\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/add_grad/tuple/group_deps\"\n",
       "  op: \"NoOp\"\n",
       "  input: \"^gradients/add_grad/Reshape\"\n",
       "  input: \"^gradients/add_grad/Reshape_1\"\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/add_grad/tuple/control_dependency\"\n",
       "  op: \"Identity\"\n",
       "  input: \"gradients/add_grad/Reshape\"\n",
       "  input: \"^gradients/add_grad/tuple/group_deps\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"_class\"\n",
       "    value {\n",
       "      list {\n",
       "        s: \"loc:@gradients/add_grad/Reshape\"\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/add_grad/tuple/control_dependency_1\"\n",
       "  op: \"Identity\"\n",
       "  input: \"gradients/add_grad/Reshape_1\"\n",
       "  input: \"^gradients/add_grad/tuple/group_deps\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"_class\"\n",
       "    value {\n",
       "      list {\n",
       "        s: \"loc:@gradients/add_grad/Reshape_1\"\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/MatMul_grad/MatMul\"\n",
       "  op: \"MatMul\"\n",
       "  input: \"gradients/add_grad/tuple/control_dependency\"\n",
       "  input: \"weight/read\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"transpose_a\"\n",
       "    value {\n",
       "      b: false\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"transpose_b\"\n",
       "    value {\n",
       "      b: true\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/MatMul_grad/MatMul_1\"\n",
       "  op: \"MatMul\"\n",
       "  input: \"x\"\n",
       "  input: \"gradients/add_grad/tuple/control_dependency\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"transpose_a\"\n",
       "    value {\n",
       "      b: true\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"transpose_b\"\n",
       "    value {\n",
       "      b: false\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/MatMul_grad/tuple/group_deps\"\n",
       "  op: \"NoOp\"\n",
       "  input: \"^gradients/MatMul_grad/MatMul\"\n",
       "  input: \"^gradients/MatMul_grad/MatMul_1\"\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/MatMul_grad/tuple/control_dependency\"\n",
       "  op: \"Identity\"\n",
       "  input: \"gradients/MatMul_grad/MatMul\"\n",
       "  input: \"^gradients/MatMul_grad/tuple/group_deps\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"_class\"\n",
       "    value {\n",
       "      list {\n",
       "        s: \"loc:@gradients/MatMul_grad/MatMul\"\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"gradients/MatMul_grad/tuple/control_dependency_1\"\n",
       "  op: \"Identity\"\n",
       "  input: \"gradients/MatMul_grad/MatMul_1\"\n",
       "  input: \"^gradients/MatMul_grad/tuple/group_deps\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"_class\"\n",
       "    value {\n",
       "      list {\n",
       "        s: \"loc:@gradients/MatMul_grad/MatMul_1\"\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"GradientDescent/update_weight/ApplyGradientDescent\"\n",
       "  op: \"ApplyGradientDescent\"\n",
       "  input: \"weight\"\n",
       "  input: \"Placeholder\"\n",
       "  input: \"gradients/MatMul_grad/tuple/control_dependency_1\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"_class\"\n",
       "    value {\n",
       "      list {\n",
       "        s: \"loc:@weight\"\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"use_locking\"\n",
       "    value {\n",
       "      b: false\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"GradientDescent/update_bias/ApplyGradientDescent\"\n",
       "  op: \"ApplyGradientDescent\"\n",
       "  input: \"bias\"\n",
       "  input: \"Placeholder\"\n",
       "  input: \"gradients/add_grad/tuple/control_dependency_1\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"_class\"\n",
       "    value {\n",
       "      list {\n",
       "        s: \"loc:@bias\"\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"use_locking\"\n",
       "    value {\n",
       "      b: false\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"GradientDescent\"\n",
       "  op: \"NoOp\"\n",
       "  input: \"^GradientDescent/update_bias/ApplyGradientDescent\"\n",
       "  input: \"^GradientDescent/update_weight/ApplyGradientDescent\"\n",
       "}\n",
       "node {\n",
       "  name: \"ArgMax/dimension\"\n",
       "  op: \"Const\"\n",
       "  attr {\n",
       "    key: \"dtype\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"value\"\n",
       "    value {\n",
       "      tensor {\n",
       "        dtype: DT_INT32\n",
       "        tensor_shape {\n",
       "        }\n",
       "        int_val: 1\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"ArgMax\"\n",
       "  op: \"ArgMax\"\n",
       "  input: \"y\"\n",
       "  input: \"ArgMax/dimension\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"Tidx\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"output_type\"\n",
       "    value {\n",
       "      type: DT_INT64\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"ArgMax_1/dimension\"\n",
       "  op: \"Const\"\n",
       "  attr {\n",
       "    key: \"dtype\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"value\"\n",
       "    value {\n",
       "      tensor {\n",
       "        dtype: DT_INT32\n",
       "        tensor_shape {\n",
       "        }\n",
       "        int_val: 1\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"ArgMax_1\"\n",
       "  op: \"ArgMax\"\n",
       "  input: \"add\"\n",
       "  input: \"ArgMax_1/dimension\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"Tidx\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"output_type\"\n",
       "    value {\n",
       "      type: DT_INT64\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"Equal\"\n",
       "  op: \"Equal\"\n",
       "  input: \"ArgMax\"\n",
       "  input: \"ArgMax_1\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_INT64\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"Cast\"\n",
       "  op: \"Cast\"\n",
       "  input: \"Equal\"\n",
       "  attr {\n",
       "    key: \"DstT\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"SrcT\"\n",
       "    value {\n",
       "      type: DT_BOOL\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"Const_1\"\n",
       "  op: \"Const\"\n",
       "  attr {\n",
       "    key: \"dtype\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"value\"\n",
       "    value {\n",
       "      tensor {\n",
       "        dtype: DT_INT32\n",
       "        tensor_shape {\n",
       "          dim {\n",
       "            size: 1\n",
       "          }\n",
       "        }\n",
       "        int_val: 0\n",
       "      }\n",
       "    }\n",
       "  }\n",
       "}\n",
       "node {\n",
       "  name: \"Mean_1\"\n",
       "  op: \"Mean\"\n",
       "  input: \"Cast\"\n",
       "  input: \"Const_1\"\n",
       "  attr {\n",
       "    key: \"T\"\n",
       "    value {\n",
       "      type: DT_FLOAT\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"Tidx\"\n",
       "    value {\n",
       "      type: DT_INT32\n",
       "    }\n",
       "  }\n",
       "  attr {\n",
       "    key: \"keep_dims\"\n",
       "    value {\n",
       "      b: false\n",
       "    }\n",
       "  }\n",
       "}\n",
       "versions {\n",
       "  producer: 26\n",
       "}"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grap.as_graph_def()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "sess = tf.Session()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "sess.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "##########\n",
      "step [100], entropy loss: [0.3853611946105957]\n",
      "1.0\n",
      "0.8951\n",
      "##########\n",
      "step [200], entropy loss: [0.5912024974822998]\n",
      "0.96875\n",
      "0.8909\n",
      "##########\n",
      "step [300], entropy loss: [0.3773052394390106]\n",
      "0.96875\n",
      "0.8933\n",
      "##########\n",
      "step [400], entropy loss: [1.3406944274902344]\n",
      "0.84375\n",
      "0.8986\n",
      "##########\n",
      "step [500], entropy loss: [0.0788169801235199]\n",
      "1.0\n",
      "0.8989\n",
      "##########\n",
      "step [600], entropy loss: [0.3849349021911621]\n",
      "0.96875\n",
      "0.8672\n",
      "##########\n",
      "step [700], entropy loss: [0.9138762950897217]\n",
      "0.9375\n",
      "0.8982\n",
      "##########\n",
      "step [800], entropy loss: [0.9349144101142883]\n",
      "0.90625\n",
      "0.8955\n",
      "##########\n",
      "step [900], entropy loss: [0.20594508945941925]\n",
      "0.96875\n",
      "0.898\n",
      "##########\n",
      "step [1000], entropy loss: [0.1732374131679535]\n",
      "1.0\n",
      "0.8899\n",
      "##########\n",
      "step [1100], entropy loss: [0.4737437963485718]\n",
      "0.90625\n",
      "0.9101\n",
      "##########\n",
      "step [1200], entropy loss: [0.25144535303115845]\n",
      "0.9375\n",
      "0.9155\n",
      "##########\n",
      "step [1300], entropy loss: [0.3161582052707672]\n",
      "0.9375\n",
      "0.9152\n",
      "##########\n",
      "step [1400], entropy loss: [0.5839414000511169]\n",
      "0.90625\n",
      "0.9096\n",
      "##########\n",
      "step [1500], entropy loss: [0.21627438068389893]\n",
      "0.9375\n",
      "0.915\n",
      "##########\n",
      "step [1600], entropy loss: [0.23908284306526184]\n",
      "0.9375\n",
      "0.9136\n",
      "##########\n",
      "step [1700], entropy loss: [0.44313308596611023]\n",
      "0.90625\n",
      "0.9127\n",
      "##########\n",
      "step [1800], entropy loss: [0.29842373728752136]\n",
      "0.9375\n",
      "0.9087\n",
      "##########\n",
      "step [1900], entropy loss: [0.30522263050079346]\n",
      "0.9375\n",
      "0.9128\n",
      "##########\n",
      "step [2000], entropy loss: [0.29670843482017517]\n",
      "0.90625\n",
      "0.9107\n",
      "##########\n",
      "step [2100], entropy loss: [0.26661521196365356]\n",
      "0.9375\n",
      "0.9124\n",
      "##########\n",
      "step [2200], entropy loss: [0.09472658485174179]\n",
      "0.96875\n",
      "0.9146\n",
      "##########\n",
      "step [2300], entropy loss: [0.34009936451911926]\n",
      "0.9375\n",
      "0.9154\n",
      "##########\n",
      "step [2400], entropy loss: [0.37424108386039734]\n",
      "0.96875\n",
      "0.9162\n",
      "##########\n",
      "step [2500], entropy loss: [0.31980541348457336]\n",
      "0.9375\n",
      "0.9149\n",
      "##########\n",
      "step [2600], entropy loss: [0.21223899722099304]\n",
      "0.90625\n",
      "0.9145\n",
      "##########\n",
      "step [2700], entropy loss: [0.42635127902030945]\n",
      "0.9375\n",
      "0.9159\n",
      "##########\n",
      "step [2800], entropy loss: [0.32863664627075195]\n",
      "0.90625\n",
      "0.9157\n",
      "##########\n",
      "step [2900], entropy loss: [0.091936856508255]\n",
      "0.96875\n",
      "0.9153\n",
      "##########\n",
      "step [3000], entropy loss: [0.10749149322509766]\n",
      "0.96875\n",
      "0.9161\n"
     ]
    }
   ],
   "source": [
    "lr = 1.0\n",
    "for step in range(3000):\n",
    "    if step > 1000:\n",
    "        lr = 0.3\n",
    "    if step > 2000:\n",
    "        lr = 0.1\n",
    "    batch_x, batch_y = mnist.train.next_batch(32)\n",
    "    _, loss = sess.run(\n",
    "        [train_step, cross_entropy],\n",
    "        feed_dict={\n",
    "            x: batch_x,\n",
    "            y: batch_y,\n",
    "            learning_rate: lr\n",
    "        })\n",
    "    if (step + 1) % 100 == 0:\n",
    "        print('#' * 10)\n",
    "        print('step [{}], entropy loss: [{}]'.format(step + 1, loss))\n",
    "        print(sess.run(accuracy, feed_dict={x: batch_x, y: batch_y}))\n",
    "        print(\n",
    "            sess.run(\n",
    "                accuracy,\n",
    "                feed_dict={\n",
    "                    x: mnist.test.images,\n",
    "                    y: mnist.test.labels\n",
    "                }))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
